
import bs4

from myspider.spider import MySpider


class Job51Spider(MySpider):

    def get_start_urls(self):
        return [(
                "https://search.51job.com/list/090200,000000,0000,00,9,99,Python%2B%25E5%25BC%2580%25E5%258F%2591,2,2.html?lang=c&stype=1&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=1&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=",
                self.parse)]

    def parse(self, response):
        bs = bs4.BeautifulSoup(response.text, features="lxml")
        jobs_list = bs.find("div", attrs={"class": "dw_table", "id": "resultList"})
        if not jobs_list:
            raise Exception("Can not find the tag")

        for item in jobs_list.find_all("div", attrs={"class": "el"}):
            try:
                title_tag = item.p.span.a.attrs["title"]
                details = list(item.children)
                company = details[1].a.attrs["title"]
                address = details[5].text
                salary_range = details[7].text
                yield {
                    "job_title": title_tag,
                    "company": company,
                    "address": address,
                    "salary": salary_range
                }
            except Exception as e:
                pass

        pages_url = bs.find("div", attrs={"class": "p_in"})
        for next_page_url in pages_url.find_all("li", attrs=None):
            try:
                yield (next_page_url.a.attrs["href"], self.parse)
            except Exception as e:
                pass


job = Job51Spider()
job.crawl()
